'''
查询网站搜索标识
包含：百度pc端，360，搜狗
Q群：170555357 
'''
from multiprocessing import Pool
import requests
from bs4 import BeautifulSoup
import re,redis
from pskpackage.db import *


pool = redis.ConnectionPool(host='地址', port=6379, password='密码', db=数据表)
#查询百度搜索标记
def BaiduSearchMark(id,url,table,field,redis_val):
    global pool
    r = redis.Redis(connection_pool=pool)
    url_site = "http://www.baidu.com/s?wd=site:%s" % (str(url))
    try:
        data = requests.get(url_site, timeout=5)
    except:
        r.lpush('mark', redis_val)  # 失败数据写入redis
        return False
    soup = BeautifulSoup(data.text, "lxml")
    name = soup.find("div", attrs={'class': 'f13'})
    if name.find("span").text:
        mark = name.find("span").text
    else:
        mark = url

    if mark:
        update_task_mark(id,mark,table,field)
        print("更新成功")
    else:
        update_task_mark(id,None,table,field)
        print("更新成功")

#查询360搜索标记
def SoSearchMark(id,url,table,field,redis_val):
    global pool
    r = redis.Redis(connection_pool=pool)
    yq_url = "https://www.sogou.com/web?query=site:%s" % (str(url))
    try:
        data = requests.get(yq_url, timeout=5)
    except:
        r.lpush('mark', redis_val)  # 失败数据写入redis
        return False

    soup = BeautifulSoup(data.text, "lxml")
    num = soup.find("p", attrs={'class': 'sr-num'})

    if num:
        update_task_mark(id,url,table,field)
        print("更新成功")
    else:
        update_task_mark(id,None,table,field)
        print("更新成功")


#查询搜狗搜索标记
def SoGouSearchMark(id,url,table,field,redis_val):
    global pool
    r = redis.Redis(connection_pool=pool)
    yq_url = "https://www.sogou.com/web?query=site:%s" % (str(url))
    try:
        data = requests.get(yq_url, timeout=5)
    except:
        r.lpush('mark', redis_val)  # 失败数据写入redis
        return False

    soup = BeautifulSoup(data.text, "lxml")
    num = soup.find("p", attrs={'class': 'sr-num'})

    if num:
        update_task_mark(id,url,table,field)
        print("更新成功")
    else:
        update_task_mark(id,None,table,field)
        print("更新成功")


if __name__ == "__main__":
    r = redis.Redis(connection_pool=pool)
    p  = Pool(2)  # 4核cpu
    while True:
        try:
            redis_val = r.rpop("mark")
            if redis_val:
                redis_val = redis_val.decode()
                task = redis_val.split('^')
                if task[2]=="baidu":
                    p.apply_async(BaiduSearchMark, args=(int(task[0]),task[1],task[3],task[4],redis_val))
                elif task[2] == "so":
                    p.apply_async(SoSearchMark, args=(int(task[0]), task[1], task[3], task[4], redis_val))
                elif task[2]=="sogou":
                    p.apply_async(SoGouSearchMark, args=(int(task[0]), task[1], task[3], task[4], redis_val))
            else:
                print("没有任务")
        except:
            pass

    p.close()  # 关闭进程池,不在接收新的任务
    p.join()  # 等待子进程全部运行完成，执行后续操作
